package com.examples.spider.processor;

import com.examples.spider.model.GitBook;
import com.examples.spider.model.GitBookCatalog;
import com.examples.spider.model.GitBookChapter;
import com.examples.spider.pipeline.GitBookPipeline;
import org.apache.commons.collections.CollectionUtils;
import org.assertj.core.util.Lists;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.scheduler.component.HashSetDuplicateRemover;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

public class BookProcessor implements PageProcessor {

    private static final String URL_SEPERATOR = "/";
    private static final String sitePath = "https://www.xxbiquge.com";
    private static final String bookNum = "11_11207";
    private static final String suffix = ".html";
    private static final String isChapterRegex = "(" + sitePath + URL_SEPERATOR + bookNum + URL_SEPERATOR + "[\\d]+" + suffix + ")";

    public void process(Page page) {

        boolean isChapter = page.getUrl().regex(isChapterRegex).match();
        if (isChapter) {
            GitBookChapter chapter = new GitBookChapter();
            chapter.setTitle(page.getHtml().xpath("//*[@id='wrapper']").xpath("//div[@class='bookname']/h1/text()").toString());
            chapter.setContent(page.getHtml().xpath("//*[@id='content']/text()").toString());
            page.putField("chapter", chapter);
        } else {
            String bookName = page.getHtml().xpath("//div[@id='list']/dl/dt/text()").toString();
            GitBook gitBook = new GitBook();
            gitBook.setBookName(bookName);
            List<Selectable> nodes = page.getHtml().xpath("//div[@id='list']/dl/dd").nodes();
            if (CollectionUtils.isNotEmpty(nodes)) {
                List<GitBookCatalog> catalogs = Lists.newArrayList();
                for (Selectable node : nodes) {
                    GitBookCatalog catalog = new GitBookCatalog();
                    catalog.setCatalogName(node.xpath("//a/text()").toString());
                    catalog.setCatalogLink(sitePath + node.xpath("//a/@href").toString());
                    page.addTargetRequest(sitePath + node.xpath("//a/@href").toString());
                    catalogs.add(catalog);
                }
                gitBook.setCatalogs(catalogs);
            }
            page.putField("catalogs", gitBook);
        }
//        page.addTargetRequests(page.getHtml().links().regex("(https://www.xbiquge6.com/11_11207/[\\d]+.html)").all());
    }

    public Site getSite() {
        Site site = Site.me().setRetryTimes(2).setSleepTime(1000);
        return site;
    }

    public static void main(String[] args) {
        Pipeline filePipeline = new GitBookPipeline("D:/走进修仙");
        Spider.create(new BookProcessor()).addUrl(sitePath + URL_SEPERATOR + bookNum + URL_SEPERATOR)
                .setScheduler(new QueueScheduler().setDuplicateRemover(new HashSetDuplicateRemover()))
                .setPipelines(Lists.newArrayList(filePipeline, new ConsolePipeline()))
                .thread(10)
                .run();
    }



}
